load('stocks.mat')
%load('tokens.mat')
fid = fopen('dictionary.txt','rt');
numTokens = 0;
while (fgets(fid) ~= -1),
    numTokens = numTokens+1;
end
numTokens=200;
%numTokens=length(tokens);
down_counter=0;
up_counter=0;
token_down=zeros(1,numTokens);
token_up=zeros(1,numTokens);
% fileoffset=10;
% filestr='translateddata/outputfile2009-06-';
% datestr='2009-06-10';
% findindex=0;
% for i=1:length(date)
%     if isequal(char(date(i)),datestr)
%         findindex=i
%     end
% end

path='nimaoutputs';
foldercontent=dir(path);
numDays=size(foldercontent,1)-3;
datestr_start=18;
datestr_length=9;

x=zeros(numDays,numTokens);

stocks_status=[];
for i=1:numDays
    filename=char(foldercontent(i+3).name);
    datestr=filename(datestr_start:datestr_start+datestr_length);
    findindex=0;
    for j=1:length(date)
        if isequal(char(date(j)),datestr)
            findindex=j;
        end
    end
    stocks_status=[stocks_status,y(findindex)];
    fullfilename=[path '/' char(foldercontent(i+3).name)]

    fid = fopen(fullfilename,'rt');
    numTweets = 0;
    while (fgets(fid) ~= -1),
        numTweets = numTweets+1;
    end
    fclose(fid);
    trainMatrix = textread(fullfilename,'%d', 50*numTweets);
    trainMatrix=trainMatrix(trainMatrix>0);
    trainMatrix=trainMatrix(trainMatrix<=numTokens);
    for j=1:length(trainMatrix)
        x(i,trainMatrix(j))=x(i,trainMatrix(j))+1;
    end
    x(i,:)=x(i,:)/sum(x(i,:));

end
threshold=sum(x)/numDays
for i=1:numDays
    x(i,:)=(x(i,:)>=threshold);
end
save(['svm_output' num2str(numTokens) '.mat'],'x','stocks_status');

% load data
% define variables
X = x;
y = 2*(stocks_status'-0.5);
C = 1;
m = size(x,1);
n = size(x,2);
% train svm using cvx
cvx_begin
    variables w(n) b xi(m)
    minimize 1/2*sum(w.*w) + C*sum(xi)
    y.*(X*w + b) >= 1 - xi;
    xi >= 0;
cvx_end

save(['svm_params' num2str(numTokens) '.mat'],'w','b','threshold');

% [y,I]=sort(log_p_token_down,'descend');
% I(1:10)
